import urllib.request
from html.parser import HTMLParser
from html.parser import HTMLParseError
import re
class HungryParser(HTMLParser):
    collect_review = False
    collect_sentimental = False
    tagname = None
    doc_id = 0
    handle = None
    star = 0
    date = False
    user = False
    def __init__(self, url, handle,review_id):
        HTMLParser.__init__(self)
        self.handle = handle
        self.doc_id = review_id
        print(review_id)
        req = urllib.request.urlopen(url.decode('utf-8'))
        content = str(req.read(),'utf-8')
        self.feed(content)
        self.collect_review = False
        self.collect_sentimental = False
        self.star = 0
        self.date = False
        self.user = False

    def handle_starttag(self, tag, attrs):
        if tag == 'li':
            for name, value in attrs:
                if name =='class' and value =='review clearfix  externalReview':
                    self.doc_id = self.doc_id+1
                    self.handle.write(bytes("<doc id=\"%d\">\n"%self.doc_id,'utf-8'))
        if tag == 'i':
            for name, value in attrs:
                if name == 'title' and value == '1.0 star rating':
                    self.star = 1
                    self.handle.write(bytes("<stars>%d</stars>\n"%self.star,'utf-8'))
                    self.star = 0
                if name == 'title' and value == '2.0 star rating':
                    self.star = 2
                    self.handle.write(bytes("<stars>%d</stars>\n"%self.star,'utf-8'))
                    self.star = 0
                if name == 'title' and value == '3.0 star rating':
                    self.star = 3
                    self.handle.write(bytes("<stars>%d</stars>\n"%self.star,'utf-8'))
                    self.star = 0
                if name == 'title' and value == '4.0 star rating':
                    self.star = 4
                    self.handle.write(bytes("<stars>%d</stars>\n"%self.star,'utf-8'))
                    self.star = 0
                if name == 'title' and value == '5.0 star rating':
                    self.star = 5
                    self.handle.write(bytes("<stars>%d</stars>\n"%self.star,'utf-8'))
                    self.star = 0
        if tag == 'li':
            for name, value in attrs:
                if name =='class' and value =='review clearfix  externalReview':
                    self.handle.write(bytes("<url>http://www.yelp.com/biz/katana-ya-san-francisco?start=%d</url>\n"%(self.doc_id-1),'utf-8'))
        if tag == 'span':
            for name, value in attrs:
                if name == 'class' and value == 'date smaller':
                    self.date = True
        if tag == 'li':
            for name, value in attrs:
                if name == 'class' and value == 'user-name':
                    self.user = True
        if tag == 'p':
            for name, value in attrs:
                if name == 'class' and value == 'review_comment ieSucks':
                    self.collect_review = True
                    self.handle.write(bytes("<review>",'utf-8'))
                    

    def handle_data(self, data):
        if self.date == True:
            self.handle.write(bytes("<date>%s</date>\n" % data,'utf-8'))
        if self.user == True:
            if any(c.isalpha() for c in data):
                self.handle.write(bytes("<user>%s</user>\n" % data,'utf-8'))
        if self.collect_review == True:
            self.handle.write(bytes("%s\n" % data,'utf-8'))
            
    def handle_endtag(self, tag):
        if tag == 'p':
            if self.collect_review == True:
                self.collect_review = False
                self.handle.write(bytes("</review>\n<polarity></polarity>\n<confidence></confidence>\n</doc>\n",'utf-8'))
        if tag == 'span':
            if self.date == True:
                self.date = False
        if tag == 'li':
            if self.user == True:
                self.user = False
            
        
handle = open("dataOutput","wb")
i = 0
while (i<2500):
    try:
            strurl = "http://www.yelp.com/biz/katana-ya-san-francisco?start=%d"%i
            hungry_parser = HungryParser(strurl.encode('utf-8'), handle,i)
            i = i + 40
            handle.flush()
    except HTMLParseError:
        i = i + 40
        continue
handle.close()
    
    


